#!/usr/bin/python

import re
import nltk
from nltk import word_tokenize, trigrams
from math import sqrt

#the one below needs to use interactive nltk.download() from terminal
#nltk.download("tokenizers/punkt/english.pickle")

# text to process
text = "\nI really like python, \nit's pretty awesome; \nbut I \nhate it sometimes. Anyway ..."

# sentence tokenizer
sentok = nltk.data.load("tokenizers/punkt/english.pickle")

# regular expression to detect english words
word_re = re.compile("^[a-z']+$")

# numeric weight for left, right contexts (word "attributes")
left_w = float(1)
right_w = float(1)

# numeric weight for the word symbol itself
sym_w = float(0.2)

# norm for all encoded vectors
norm = sqrt(left_w*2 + sym_w**2 + right_w**2)

# sentence to trigrams mapping (also returns vocabulary found)
def sent2trigs(s):
#    print("sentence = %s" % s)
    tokens = word_tokenize(s)
#    print("tokens = %s" % tokens)
    is_word = lambda s: word_re.match(s) != None
    words = filter(is_word, [w.lower() for w in tokens])
#    print("words = %s" % words)
    trigs = trigrams(words)
#    print("trigrams = %s" % trigs)
    vocab = sorted(set(words))
#    print("vocab = %s" % vocab)
    return (vocab,trigs)

# encode each trigram as an sparse vector, having symbol, left and right 
# encodings; the 3 of them represented by entries of the form 
# (word_index, normal_weight). so this function returns a list of 3 tuples.
# the first parameter is a hash mapping each word to its index
def encode(vocab, trig):
    left = trig[0]
    word = trig[1]
    right = trig[2]
    return [(vocab[word], sym_w/norm), 
            (vocab[left], left_w/norm), 
            (vocab[right], right_w/norm)]
 
#main
accum = lambda t1,t2: (t1[0]+t2[0], t1[1]+t2[1])
(vocab,trigs) = reduce(accum, [sent2trigs(s) for s in sentok.tokenize(text)])
vocab = dict(zip(vocab, xrange(len(vocab))))
encoding = map(lambda t: encode(vocab,t), trigs)
print("total vocab = %s" % vocab)
print("total trigrams = %s" % trigs)
print("encoding  = %s" % encoding)
pp_v = lambda v: " ".join(map(lambda t: "%6d %.6f" % t, v))
print("pp encoding  = \n%s" % "\n".join(map(pp_v, encoding)))
